"""
Copyright (c) 2022 Huawei Technologies Co.,Ltd.

openGauss is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:

          http://license.coscl.org.cn/MulanPSL2

THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details.
"""
"""
Case Type   : 故障&可靠性测试
Case Name   : 事务提交方式不为remote_apply,备节点注入磁盘满故障，清除故障时间大于延迟时间
Create At   : 2021/03/01
@zou_jialiang0505328126
Description :
    1.备节点切换为级联备
    2.配置级联节点备份延迟为1min
    3.创建表，并插入数据
    4.级联备注入磁盘满故障
    5.5s/60s查询备节点
    6.恢复故障，重启集群
    7.查询级联备
    8.删除数据
    9.备节点注入磁盘满
    10.5s/60s查询级联备
    11.恢复故障，重启集群
    12.查询备
    13.插入数据
    14.级联备注入磁盘满
    15.5s后清除故障，重启集群
    16.查询级联备及备节点
    17.删除数据
    18.备节点注入磁盘满故障
    19.5s后清除故障，重启集群
    20.5s/60s查询备及级联备
Expect      :
    1.切换成功
    2.配置成功
    3.创建表成功
    4.注入故障成功
    5.5s备未同步，60s后同步
    6.清除故障成功，重启成功
    7.级联备同步成功
    8.删除成功
    9.故障注入成功
    10.5s级联备未同步，60s后同步
    11.清除故障成功，重启成功
    12.备同步
    13.插入数据成功
    14.注入故障成功
    15.清除故障成功，重启集群可能为Degrade
    16.若为Degrade等待60s，级联备及备同步，若start成功，此时已同步
    17.删除数据成功
    18.注入故障成功
    19.清除故障成功，重启集群可能为Degrade
    20.若为Degrade等待60s，级联备及备同步，若start成功，此时已同步
History     :
    Modified by @wan005 2021/11/2 : 优化用例 更换级联备一致等待方法
    Modified by @wan005 2022/04/15 : 增加循环查询等待主备一致
"""

import unittest
import os
import time
from yat.test import macro
from yat.test import Node
from testcase.utils.CommonSH import CommonSH
from testcase.utils.Common import Common
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger


class RecoveryDelay(unittest.TestCase):
    commonshpri = CommonSH('PrimaryDbUser')
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info('----Opengauss_Reliability_Dbsys_Case246.py 开始执行------')
        self.db_primary_root_node = Node(node='PrimaryRoot')
        self.db_primary_user_node = Node(node='PrimaryDbUser')
        self.comsta = [Common('Standby1Root'), Common('Standby2Root')]
        self.db_standby_node =  \
            [Node(node='Standby1Root'), Node(node='Standby2Root')]
        self.comshsta = \
            [CommonSH('Standby1DbUser'), CommonSH('Standby2DbUser')]
        self.constant = Constant()
        self.db_disk_name = []
        self.tb_name = 'tb_241'
        self.conf_path = os.path.join(
            macro.DB_INSTANCE_PATH, macro.DB_PG_CONFIG_NAME)

        self.log.info('------同步集群时间--------')
        current = self.db_primary_root_node.sh(
            "date \"+%m/%d/%Y %H:%M:%S\"").result()
        self.log.info(current)
        datecmd = f'date -s "{current}"'
        self.log.info(datecmd)
        for i in range(2):
            result = self.db_standby_node[i].sh(datecmd).result()
            self.log.info(result)

        self.log.info('================获取备节点磁盘名=========')
        for i in range(2):
            self.db_disk_name.append(
                self.comsta[i].get_disk_name(macro.DB_INSTANCE_PATH))

    def test_recoverydelay(self):
        self.log.info('--------创建表-------')
        sql = f"drop table if exists {self.tb_name};" \
            f"create table {self.tb_name}(i int, s char(10));"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.assertIn(self.constant.TABLE_CREATE_SUCCESS, result)

        self.log.info('-------将备节点1切换为级联备---------')
        result = self.comshsta[1].buildStandby('-M cascade_standby')
        self.assertIn(self.constant.REBUILD_SUCCESS_MSG, result)
        result = self.comshsta[1].exec_refresh_conf()
        self.assertTrue(result)

        self.log.info('-----设置recovery_min_apply_delay=1min----')
        result = self.commonshpri.executeGsguc(
            'reload', self.constant.GSGUC_SUCCESS_MSG,
            'recovery_min_apply_delay=1min')
        self.assertTrue(result)

        self.log.info('-----------查询参数-----------')
        result = self.commonshpri.executDbSql(
            'show recovery_min_apply_delay;')
        self.log.info(result)
        self.assertIn('1min', result)

        self.log.info('--------等待主备一致------------')
        for i in range(90):
            result = self.comshsta[0].check_cascade_standby_consistency()
            if result:
                break
            time.sleep(20)
        self.assertTrue(result)
        result = self.comshsta[1].check_data_consistency()
        self.assertTrue(result)

        self.log.info('------------插入数据------------')
        sql = f"insert into {self.tb_name} values(1,'test');"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.assertIn('INSERT', result)

        self.log.info('-------------级联备注入磁盘满---------------')
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe ' \
            f'"inject  rfile_full (diskname) values  ({self.db_disk_name[1]})"'
        self.log.info(cfedisk_cmd)
        result = self.db_standby_node[1].sh(cfedisk_cmd).result()
        self.log.info(result)

        self.log.info('----------等待5s查询备再等待60s后查询备---------------')
        time.sleep(5)
        sql = f"select sysdate;select * from {self.tb_name};"
        result = self.comshsta[0].executDbSql(sql)
        self.log.info(result)
        self.assertIn('(0 rows)', result)
        time.sleep(70)
        result = self.comshsta[0].executDbSql(sql)
        self.log.info(result)
        self.assertIn('1 | test', result)

        self.log.info('----------------------清除故障----------------------')
        cfedisk_cmd = f'''{macro.CFE_PATH}/cfe "clean  
                rfile_full  where(diskname={self.db_disk_name[1]})"'''
        tmp = self.db_standby_node[1].sh(cfedisk_cmd).result()
        self.log.info(tmp)

        time.sleep(5)
        self.log.info('----------重启数据库-----------')
        result = self.commonshpri.stopDbCluster()
        self.log.info(result)
        result = self.commonshpri.startDbCluster()
        self.assertTrue(result)

        self.log.info('----------查询级联备---------------')
        result = self.comshsta[1].executDbSql(sql)
        self.log.info(result)
        self.assertIn('1 | test', result)

        self.log.info('---------------删除数据------------------')
        sql = f"delete from {self.tb_name};"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.assertIn('DELETE', result)

        self.log.info('-------------备节点注入故障---------------')
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe ' \
            f'"inject  rfile_full (diskname) ' \
            f'values  ({self.db_disk_name[0]})"'
        self.log.info(cfedisk_cmd)
        result = self.db_standby_node[0].sh(cfedisk_cmd).result()
        self.log.info(result)

        self.log.info('----------等待5s查询备再等待60s后查询级联备---------------')
        time.sleep(5)
        sql = f"select sysdate;select * from {self.tb_name};"
        result = self.comshsta[1].executDbSql(sql)
        self.log.info(result)
        self.assertIn('1 | test', result)
        time.sleep(70)
        result = self.comshsta[1].executDbSql(sql)
        self.log.info(result)
        self.assertIn('(0 rows)', result)

        self.log.info('--------------清除故障----------------------')
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe "clean rfile_full ' \
            f'where(diskname={self.db_disk_name[0]})"'
        tmp = self.db_standby_node[0].sh(cfedisk_cmd).result()
        self.log.info(tmp)

        self.log.info('----------重启数据库-----------')
        time.sleep(5)
        result = self.commonshpri.stopDbCluster()
        self.log.info(result)
        result = self.commonshpri.startDbCluster()
        self.assertTrue(result)

        self.log.info('----------查询备节点-------------------')
        time.sleep(5)
        sql = f"select sysdate;select * from {self.tb_name};"
        result = self.comshsta[0].executDbSql(sql)
        self.log.info(result)
        self.assertIn('(0 rows)', result)

        self.log.info('--------插入数据-------')
        sql = f"insert into {self.tb_name} values(1,'test');"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.assertIn('INSERT', result)

        self.log.info('-------------级联备注入磁盘满---------------')
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe ' \
            f'"inject  rfile_full (diskname) values  ({self.db_disk_name[1]})"'
        self.log.info(cfedisk_cmd)
        result = self.db_standby_node[1].sh(cfedisk_cmd).result()
        self.log.info(result)

        self.log.info('----------------------清除故障----------------------')
        time.sleep(5)
        cfedisk_cmd = f'''{macro.CFE_PATH}/cfe "clean  
                        rfile_full  where(diskname={self.db_disk_name[1]})"'''
        tmp = self.db_standby_node[1].sh(cfedisk_cmd).result()
        self.log.info(tmp)

        time.sleep(5)
        self.log.info('----------重启数据库-----------')
        result = self.commonshpri.stopDbCluster()
        self.log.info(result)
        start_result = self.commonshpri.startDbCluster(True)
        flg = 'Degrade' in start_result or \
              self.constant.START_SUCCESS_MSG in start_result
        self.assertTrue(flg)

        self.log.info('----------查询级联备/备---------------')
        sql = f"select sysdate;select * from {self.tb_name};"
        if 'Degrade' in start_result:
            time.sleep(70)
        for i in range(2):
            result = self.comshsta[i].executDbSql(sql)
            self.log.info(result)
            self.assertIn('1 | test', result)

        self.log.info('---------------删除数据------------------')
        sql = f"delete from {self.tb_name};"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.assertIn('DELETE', result)

        self.log.info('-------------备节点注入故障---------------')
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe ' \
            f'"inject  rfile_full (diskname) ' \
            f'values  ({self.db_disk_name[0]})"'
        self.log.info(cfedisk_cmd)
        result = self.db_standby_node[0].sh(cfedisk_cmd).result()
        self.log.info(result)

        self.log.info('--------------5s清除故障----------------------')
        time.sleep(5)
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe "clean rfile_full ' \
            f'where(diskname={self.db_disk_name[0]})"'
        tmp = self.db_standby_node[0].sh(cfedisk_cmd).result()
        self.log.info(tmp)

        self.log.info('----------重启数据库-----------')
        time.sleep(5)
        result = self.commonshpri.stopDbCluster()
        self.log.info(result)
        start_result = self.commonshpri.startDbCluster(True)
        flg = 'Degrade' in start_result or \
              self.constant.START_SUCCESS_MSG in start_result
        self.assertTrue(flg)

        self.log.info('----------查询级联备/备---------------')
        sql = f"select sysdate;select * from {self.tb_name};"
        for i in range(2):
            result = self.comshsta[i].executDbSql(sql)
            self.log.info(result)
            if 'Degrade' in start_result:
                self.assertIn('1 | test', result)
            else:
                self.assertIn('(0 rows)', result)
        if 'Degrade' in start_result:
            time.sleep(70)
            for i in range(2):
                result = self.comshsta[i].executDbSql(sql)
                self.log.info(result)
                self.assertIn('(0 rows)', result)

    def tearDown(self):
        self.log.info('-----------------------------环境清理--------------------')
        self.log.info('----------------------清除故障---------------------------')
        for i in range(2):
            cfedisk_cmd = f'''{macro.CFE_PATH}/cfe "clean 
            rfile_full  where(diskname={self.db_disk_name[i]})"'''
            tmp = self.db_standby_node[i].sh(cfedisk_cmd).result()
            self.log.info(tmp)

        self.log.info('--------还原集群-------')
        self.comshsta[1].buildStandby('-M standby')
        result = self.comshsta[1].exec_refresh_conf()
        self.log.info(result)

        self.log.info('--------------还原配置文件-------------')
        shell_cmd = f"rm -rf {self.conf_path};" \
            f"cp {self.conf_path}_testbak {self.conf_path};" \
            f"rm -rf {self.conf_path}_testbak"
        self.log.info(shell_cmd)
        result = self.db_primary_user_node.sh(shell_cmd).result()
        self.log.info(result)

        self.log.info('-------------还原recovery_min_apply_delay----------')
        result = self.commonshpri.executeGsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            'recovery_min_apply_delay=0')
        self.log.info(result)

        self.log.info('-----------重启数据库-----------')
        result = self.commonshpri.stopDbCluster()
        self.log.info(result)
        result = self.commonshpri.startDbCluster()
        self.log.info(result)

        time.sleep(5)
        self.log.info('--------删除表-------')
        sql = f"drop table if exists {self.tb_name};"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.log.info('---Opengauss_Reliability_Dbsys_Case246.py 执行结束-------')